/* NoX (NoC Simulator)
 *
 * Dept. of Computer Science & Engineering, Pennsylvania State University.
 * All Rights Reserved.
 *  
 * 1. License     
 * NoX is distributed free of charge for academic, educational, noncommercial 
 * research purposes as long as this notice in its entirety is preserved in
 * every file included in this package.
 * All commercial use of this program requires separate licence. Contact the
 * author for details.
 * 
 * 2. All the publications that used the simulation results generated by the 
 * NoX should notify the author of the publication information and put 
 * following reference.
 *
 *  http://www.cse.psu.edu/~dpark/nox/
 * 
 * 3. Modification of the source code is permitted and encouraged as long as 
 * it follows the terms described in this copyright notice.
 *
 * 4. The author is not responsible for any problems caused by possible errors
 * of the NoX package. Therefore, users should verify the simulation result
 * before using it in their publication.
 *
 * Dept. of Computer Science & Engineering, Pennsylvania State University.
 * Contact: dpark@cse.psu.edu 
 * 
 * 6. If problems are found with the NoX package, please send an email to the
 * author for discussion and correction.

*/

/* Update History
 *
 * Jan. 31, 2006  Version 1.0 released by Dongkook Park 
 *
 */

/* NIC.C - Network Interface Controller (message injection/ejection) */

#include <stdio.h>
#include <stdlib.h>
#include <cassert>
#include "main.h"
#include "router_common.h"
#include "router.h"
#include "flit.h"
#include "flit_move.h"
#include "app.h"
#include "nic.h"
#include "shared.h"
#include "batch.h"
#include "rank.h"

#ifdef TR_INTEG
// TR_INTEG begin
#undef INVALID
#include "defines.h"
#include "transaction.h"
#include "globals.h"
#undef INVALID
#define INVALID 1
// TR_INTEG end
#endif

#ifndef TR_INTEG
#include <vector>
struct nic_entry_t {
  int src;
  int dst;
};
extern std::vector<nic_entry_t> nic_queue;
#endif

extern std::vector<source_queue_t> source_queue[MAX_NODES]; 

int last_selected_rinbuf_vc[MAX_NODES][MAX_PC]={0};
int is_usable_vc[MAX_VC]={YES};
void nic_output_buf_to_router_input_buf_stage()
{
  // Move message in NIC output buf to router input buf.
  static int last_selected_vc[MAX_NODES][MAX_NIC_PC]={0};
  int node, pc, pc_index, i, vc_candidate=-1, vc_nic_candidate=-1;
  flit_t *flit_ptr;

  for(node=0; node<NUM_NODES; node++)
  {
    NUM_PC     = router_info[node].num_pc;
    NUM_NIC_PC = router_info[node].num_nic_pc;
    NUM_VC     = router_info[node].num_vc;
    for(pc=0; pc<NUM_NIC_PC; pc++)
    {
      // This pc_index is for the router input buffer.
      // The index of the nic_output_buf begins from '0' to 'NUM_NIC_PC-1' while
      // the index of the router_input_buf begins from 'NUM_PC-NUM_NIC_PC' to 
      // 'NUM_PC-1'. Therefore, we need to add 'NUM_PC-NUM_NIC_PC' when accessing
      // the router_input_buf array.
      pc_index = pc + NUM_PC - NUM_NIC_PC;
      get_best_nic_vc(node, pc, &vc_nic_candidate, &vc_candidate);
      if(vc_nic_candidate == -1)
        continue;
      
      read_flit(&(nic_output_buf[node][pc][vc_nic_candidate]), &flit_ptr);
      if(HEAD_FLIT)
      {
        rinbuf_info[node][pc][vc_candidate].vc_stat = VC_BUSY;
        vc_nic_info[node][pc][vc_nic_candidate].out_vc = vc_candidate;
        vc_nic_info[node][pc][vc_nic_candidate].vc_stat = VC_BUSY;
        //This marks the time when this flit became the head of nic queue
        if(flit_ptr->entry_time == flit_ptr->msg_inj_tm)
          flit_ptr->entry_time = sim_clock;
      }
      else
      {
        vc_candidate = vc_nic_info[node][pc][vc_nic_candidate].out_vc;
      }
      if(TAIL_FLIT)
      {
        rinbuf_info[node][pc][vc_candidate].vc_stat = VC_IDLE;
        vc_nic_info[node][pc][vc_nic_candidate].vc_stat = VC_IDLE;
      }

      // We found a VC with data. Send the data to the router input buffer.
      recv_flit(&(nic_output_buf[node][pc][vc_nic_candidate]), &flit_ptr);

      if(sql == YES)
        fprintf(fsql, "INSERT INTO FLOW VALUES(0, %d,%d,%d,%d,%d,%d);\n", 
            flit_ptr->flit_num, node, pc_index, vc_nic_candidate, 0, sim_clock);
      if(verbose == YES) 
      { 
        printf("rinbuf  [%d][%d][%d]-flit:%d(%s) ninbuf [%d][%d][%d] queue delay :%d interference:%d\n", node, pc_index, vc_candidate, 
            flit_ptr->flit_num, (HEAD_FLIT)? "HEAD": 
            (TAIL_FLIT)? "TAIL":"MIDDLE", node, pc, vc_nic_candidate,
            sim_clock - flit_ptr->msg_inj_tm, flit_ptr->interference_cycles);
        fflush(stdout);
      }

      // Update flit injection time.
      flit_ptr->inj_tm = sim_clock; 
      flit_ptr->entry_time = sim_clock;

      if(verbose == YES)
        printf("Setting entry time [%d][%d][%d]-flit:%d(%s) : %lld\n",node, pc_index, vc_candidate, 
            flit_ptr->flit_num, (HEAD_FLIT)? "HEAD": 
            (TAIL_FLIT)? "TAIL":"MIDDLE", (long long)sim_clock);

      //Power Update
      if(router_info[node].type == LOCAL)
        p_buffer += 0.0;
      else
        p_buffer += PD_BUFFER;
      if(topology == BFLY || (concentrated == YES && hybrid_topology == NO))
        p_link += 0.25*PD_LINK;

      // If any of soft error detection/recovery scheme is used, we need to double 
      // the power consumption of the router input buffer since  double-buffering 
      // scheme is used (for router buffer error).
      if(link_err_rate + routing_err_rate + swarbiter_err_rate > 0) 
        p_buffer += PD_BUFCTR + PD_BUFFER;

      send_flit(&(router_input_buf[node][pc_index][vc_candidate]), &flit_ptr);

      // Keep track of the activity.
      if(sim_clock > warmup_cycle)
        total_activity[node]++;

      // Add buffer power.
      p_buffer += PD_BUFCTR + PD_BUFFER;
      last_selected_vc[node][pc] = vc_nic_candidate;
    }// for pc
  }// for node
}

void nic_input_buf_to_PE_stage()
{
  // This stage handles message ejection in NIC.

  static int    last_selected_vc[MAX_NODES][MAX_NIC_PC] = {0};
  static int    error[MAX_NODES][MAX_NIC_PC][MAX_VC] = {0};
  int last_vc, node, pc, i, vc_candidate, delay;
  flit_t *flit_ptr;

  for(node=0; node<NUM_NODES; node++)
  {
    NUM_PC     = router_info[node].num_pc;
    NUM_NIC_PC = router_info[node].num_nic_pc;
    NUM_VC     = router_info[node].num_vc;
    for(pc=0; pc<NUM_NIC_PC; pc++)
    {
      last_vc = last_selected_vc[node][pc];
      for(i=1; i<=NUM_VC; i++)
      {
        vc_candidate = (last_vc + i) % NUM_VC;

        if( msg_cnt(&(nic_input_buf[node][pc][vc_candidate])) > 0 )
        {
          read_flit(&(nic_input_buf[node][pc][vc_candidate]), &flit_ptr);

          //--------------------------------------------------------------------------------------
          // Check whether this has go through the error penalty check.
          if(flit_ptr->blk_chk_done == NO)
          {
            //------------------------------------------------------
            // Check for the error
            if(retrans_type != NONE)
            {
              // In FEC or HFEC, need to "CORRECT" errors.
              if(retrans_type == FEC || retrans_type == HFEC) { p_err_chk += PD_SECDED; }

              // In other policies, need to "CHECK" errors.
              if(retrans_type != FEC) 
              { 
                p_err_chk += PD_CRCCHK;

                // Now, check for the LINK error.
                if( (flit_ptr->error & ERROR_LINK) > 0 )
                {
                  if(retrans_type == E2E || 
                      (retrans_type == HE2E && !HEAD_FLIT))
                  {
                    error[node][pc][vc_candidate] = YES;
                    delay = 0;
                  }// E2E & HE2E

                  //---------------------------------------------------------------------
                  // Actually, this should not happen since we're assuming that the soft 
                  // error occurs only in the inter-router link as for now. Anyway, leave
                  // this here for further extension.
                  else if(  retrans_type == HBH || 
                      (retrans_type == HFEC && HEAD_FLIT) || 
                      (retrans_type == HE2E && HEAD_FLIT)  )
                  {
                    if(HEAD_FLIT)
                      delay=3; 
                    else if(TAIL_FLIT)
                      delay=3; 
                    else
                      delay=(flit_ptr->pos == flit_ptr->msglen/*MSG_LEN*/-1)? 3:2; 
                    // add buffer read power (retransmitting flit) and retransmission control 
                    // logic power.
                    p_retrans += PD_BUFFER + PD_HBHCTR;
                  }// else if

                  flit_ptr->error &= ~ERROR_LINK; // Mark as error-free.
                  //---------------------------------------------------------------------
                }// (flit_ptr->error & ERROR_LINK) > 0 
                else
                  delay = 0;
              }// else

              flit_ptr->error_penalty = delay;
            }// if (!NONE)
            // End of error check
            //------------------------------------------------------

          }// if blk_chk_done

          // If this flit has error penalty related to it, block this flit.
          if(flit_ptr->error_penalty >0)
          {
            flit_ptr->blk_chk_done = YES;
            flit_ptr->error_penalty--;
            continue;
          }

          // Now, this flit can proceed...
          flit_ptr->blk_chk_done = NO;// Reset this to NO. 
          //--------------------------------------------------------------------------------------

          // Handle retransmission 
          if(retrans_type == E2E || (retrans_type == HE2E && !HEAD_FLIT))
          {
            if(TAIL_FLIT && error[node][pc][vc_candidate] == YES)
            {
              // If an error is detected, send NACK to the source node.
#ifdef TR_INTEG 
              app_to_nic_output_buf_stage(flit_ptr->data.dnode, flit_ptr->data.snode , YES, flit_ptr->trid, flit_ptr->msglen);
#else
#ifdef TRACE_READ
              app_to_nic_output_buf_stage(flit_ptr->data.dnode, flit_ptr->data.snode , YES, -1, flit_ptr->msglen);
#else
              app_to_nic_output_buf_stage(flit_ptr->data.dnode, flit_ptr->data.snode , YES, -1, MSG_LEN);
#endif
#endif
              num_e2e_nack++;
              error[node][pc][vc_candidate] = NO;
            }

            // If this flit is a NACK, retransmit packet.
            if(TAIL_FLIT && flit_ptr->is_nack == YES)
            {
#ifdef TR_INTEG
              app_to_nic_output_buf_stage(flit_ptr->data.dnode, flit_ptr->data.snode , NO, flit_ptr->trid, flit_ptr->msglen);
#else
#ifdef TRACE_READ
              app_to_nic_output_buf_stage(flit_ptr->data.dnode, flit_ptr->data.snode , NO, -1, flit_ptr->msglen);
#else
              app_to_nic_output_buf_stage(flit_ptr->data.dnode, flit_ptr->data.snode , NO, -1, MSG_LEN);
#endif
#endif
              num_e2e_retrans++;
            }
          }

          // We found a VC with data. Transfer that data from the NIC input buf to the PE.
          recv_flit(&(nic_input_buf[node][pc][vc_candidate]), &flit_ptr);
          last_selected_vc[node][pc] = vc_candidate;

          num_ejt_flit++;
          if(flit_ptr->short_flit == YES)
            short_flits_sum++; 


          if(flit_ptr->msgtype == CONTROL)
            control_ejt_flit++;
          else
            data_ejt_flit++;


          //=======================================================================
          //Stall Time Fairness Support
          //=======================================================================
          //Account for addional cycle might have incurred from nic to rinbuf
          //when injecting the packet
          //flit_ptr->interference_cycles -= (flit_ptr->pos + 1);

          /*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*/
          // start batching support
          /*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*/
          //local batching book keeping
          if(batching_mode[node] == BATCHING_LOCAL_FULLBATCHING)
            if(flit_ptr->batch_id == current_batch[node])
            {
              local_source_jobs_done[node]++;
              if(verbose == YES)
                printf("Updating local_source_jobs_done:%d for flit:%d at nic\n",local_source_jobs_done[node],flit_ptr->flit_num);
            }
          /*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*/
          // end batching support
          /*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*/
#ifdef TR_INTEG
          if(HEAD_FLIT)
          {
            if(flit_ptr->priority_id != -1)
              processor[flit_ptr->priority_id].priority_inversion_latency += flit_ptr->priority_inversion_cycles;
          }
#endif
          if(TAIL_FLIT)
          {

            if(flit_ptr->msgtype == CONTROL)
            {
#ifndef TR_INTEG
              if(RESP_RQ_TRAFFIC)
              {
                int snode = flit_ptr->data.dnode;
                int dnode = flit_ptr->data.snode;
                if(two_networks==YES)
                {
                  snode = snode+NUM_NODES/2;
                  dnode = dnode+NUM_NODES/2;
                }
                nic_entry_t tmp;
                if(verbose == YES)
                  printf("inject response [%d -> %d : len %d at %d ]\n",flit_ptr->data.dnode,flit_ptr->data.snode,MSG_LEN,sim_clock);
                if(!app_to_nic_output_buf_stage(snode,dnode, NO, -1, MSG_LEN))
                {
                  tmp.src = flit_ptr->data.dnode;
                  tmp.dst = flit_ptr->data.snode;
                  nic_queue.push_back(tmp);
                  if(verbose == YES)
                    printf("dropped response\n");
                }
              }
#endif
              control_ejt_msg++;
            }
            else
            {
              if(hybrid_topology == YES)
              {
                num_ejt_flit += flit_ptr->msglen - flit_ptr->llen;
              }
              data_ejt_msg++;
            }

            total_packet_delay_cycle += sim_clock - flit_ptr->msg_inj_tm;
            total_packet_queue_cycle += (flit_ptr->inj_tm - flit_ptr->msg_inj_tm/* - flit_ptr->pos*/);
            total_bus_delay_cycle += flit_ptr->bus_latency;
            /*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*/
            //Source Queue logic
            source_job_delay[flit_ptr->data.snode] += sim_clock - flit_ptr->msg_inj_tm;
            if(source_queue_logic_enabled)
              source_queue_update(flit_ptr->data.snode,flit_ptr->packet_id,true);

            /*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*/
            // start batching support
            /*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*/
            //batching support
            if(batching_mode[flit_ptr->data.snode] > BATCHING_MODE_NONE)
            {
              //global batching book keeping
              if(batching_mode[flit_ptr->data.snode] > BATCHING_MODE_GLOBAL)
                if(flit_ptr->batch_id == current_batch[flit_ptr->data.snode])
                {
                  global_source_jobs_done[flit_ptr->data.snode]++;
                  if(verbose == YES)
                    printf("node:%d count:%d flit num:%d clk:%lld inj_tm:%d sq size:%d\n",
                        flit_ptr->data.snode,global_source_jobs_done[flit_ptr->data.snode],flit_ptr->flit_num,sim_clock,flit_ptr->msg_inj_tm,source_queue[flit_ptr->data.snode].size());
                }
            }
            /*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*/
            // end batching support
            /*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*/

            num_ejt_msg++;
            if(hybrid_topology == YES && flit_ptr->data.snode == flit_ptr->data.dnode)
              l_msgs++;
            if(hybrid_topology == YES && flit_ptr->data.snode != flit_ptr->data.dnode)
              g_total_num_hop += flit_ptr->num_hop;

#ifdef TR_INTEG
            // TR_INTEG begin
            if(flit_ptr->trid != -1)
            {
              TRANSTYPE *tr = map_transaction(flit_ptr->trid);
              if(verbose == YES)
              {
                printf("Informing transaction :%d\n",tr->tr_id);
                tr->print();
              }
              tr->icn_trans_comp = true; 
              sim_end_flag = VALID; // End simulation.
              icn_event_flag = true;
              if(flit_ptr->msgtype == CONTROL)
              {
                if(inverse_cache_network_layout[flit_ptr->data.dest] != -1)
                  processor[tr->tid].per_flow_packets[inverse_cache_network_layout[flit_ptr->data.dest]]++;
                //processor[inverse_cpu_network_layout[flit_ptr->data.src]].per_flow_packets[inverse_cache_network_layout[flit_ptr->data.dest]]++;
              }
              if(verbose == YES) 
                printf("Eject  [%d][%d][%d]-flit:%d, node:%d -> dnode:%d src_PE:%d -> dst_PE:%d [%d] at clock:%lld type:%s\n", node, pc+NUM_PC-NUM_NIC_PC, 
                    vc_candidate, flit_ptr->flit_num, flit_ptr->data.snode, flit_ptr->data.dnode, flit_ptr->data.src, flit_ptr->data.dest,inverse_cache_network_layout[flit_ptr->data.dest],
                    (long long)sim_clock, flit_ptr->msgtype == CONTROL ? "C":"D"); fflush(stdout);

              processor[tr->tid].hop_count += flit_ptr->num_hop;
              processor[tr->tid].packet_latency += sim_clock-flit_ptr->msg_inj_tm;
              processor[tr->tid].blocking_cycles += flit_ptr->delay_cycle;
              processor[tr->tid].buff_blocking_cycles += flit_ptr->buff_full_interference_cycles;
              if(flit_ptr->msgtype == CONTROL)
              {
                processor[tr->tid].ctrl_packet_latency += sim_clock-flit_ptr->msg_inj_tm;
                processor[tr->tid].ctrl_packets++;
              }
              else
              {
                processor[tr->tid].data_packet_latency += sim_clock-flit_ptr->msg_inj_tm;
                processor[tr->tid].data_packets++;
              }

              int zero_load_cycles = flit_ptr->num_hop*(arch + 1) + 2 + flit_ptr->msglen;

              if(tr->mem_op_id != -1)
                processor[tr->tid].rob_array[tr->rob_entry].hops += flit_ptr->num_hop + 1;
              if(tr->mem_op_id != -1)
                assert(tr->rob_entry < 256 && tr->rob_entry >=0);

              if(NUM_NODES != NUM_PE && flit_ptr->data.snode == flit_ptr->data.dnode)
                processor[tr->tid].local_packets++;
              if(NUM_NODES == NUM_PE && flit_ptr->num_hop <= 1)
                processor[tr->tid].local_packets++;
            }
#endif
          }

          //assert(flit_ptr->interference_cycles==0);


          /*if(HEAD_FLIT)
            {
            int zero_load_cycles = flit_ptr->num_hop*(arch + 1) + 2 + flit_ptr->msglen //one msglen for serialization, one for queueing from nic->router
            + flit_ptr->interference_cycles;
            int packet_cycles = sim_clock - flit_ptr->msg_inj_tm;
            if(packet_cycles - zero_load_cycles > 30)
            {
            printf("nic->PE [%d][%d][%d]-flit:%d(%s) source:%d ejt msgs:%d ejt flits:%d inj msgs:%d lat:%lld inj_tm:%d\n", node, pc, vc_candidate, 
            flit_ptr->flit_num, (HEAD_FLIT)? "HEAD": (TAIL_FLIT)? "TAIL":"MIDDLE",
            flit_ptr->data.snode, num_ejt_msg,num_ejt_flit,num_inj_msg, sim_clock - flit_ptr->inj_tm, flit_ptr->msg_inj_tm);

            printf("ic:%d sa:%d buff:%d bc:%d nc:%d qc:%d tc:%d zl:%d h:%d len:%d\n",
            flit_ptr->interference_cycles,flit_ptr->sa_cycles,flit_ptr->buff_full_interference_cycles,flit_ptr->delay_cycle, 
            sim_clock-flit_ptr->inj_tm, flit_ptr->inj_tm - flit_ptr->msg_inj_tm,sim_clock-flit_ptr->msg_inj_tm,zero_load_cycles,flit_ptr->num_hop,flit_ptr->msglen);
            exit(1);
            }
            }*/
          //if(flit_ptr->msgtype == CONTROL)
          //if(flit_ptr->flit_type == HEAD_FLIT)
          {
            total_delay_cycle       += (flit_ptr->delay_cycle);
            if(flit_ptr->inj_tm - flit_ptr->msg_inj_tm - flit_ptr->pos>0)
              total_queueing_latency  += (flit_ptr->inj_tm - flit_ptr->msg_inj_tm - flit_ptr->pos);
            total_network_latency   += (sim_clock - flit_ptr->inj_tm + flit_ptr->e2e_penalty); 
            //                     + (float)(MSG_LEN-1)/(float)MSG_LEN;
            total_latency           = total_queueing_latency + total_network_latency;

            if(TAIL_FLIT)
              total_num_hop          += flit_ptr->num_hop;



            //printf("%d %d %d\n",flit_ptr->num_hop, flit_ptr->data.snode, flit_ptr->data.dnode);

            if(flit_ptr->num_hop <= 1)
              flits_one_hop++;
            else if(flit_ptr->num_hop <= 2)
              flits_two_hop++;
            else if(flit_ptr->num_hop <= 3)
              flits_three_hop++;
            else
              flits_three_plus_hop++;



            if(flit_ptr->delay_cycle > 20)
              total_aged_flit20++;
            else if(flit_ptr->delay_cycle > 10)
              total_aged_flit10++;

            if(flit_ptr->msgtype == CONTROL)
            {
              cmsg_total_delay_cycle       += /*(float)*/(flit_ptr->delay_cycle);
              cmsg_total_queueing_latency  += /*(float)*/(flit_ptr->inj_tm - flit_ptr->msg_inj_tm /*- flit_ptr->pos*/);
              cmsg_total_network_latency   += /*(float)*/(sim_clock - flit_ptr->inj_tm + flit_ptr->e2e_penalty); 
              cmsg_total_latency           = cmsg_total_queueing_latency + cmsg_total_network_latency;
            }
            else
            {
              dmsg_total_delay_cycle       += /*(float)*/(flit_ptr->delay_cycle);
              dmsg_total_queueing_latency  += /*(float)*/(flit_ptr->inj_tm - flit_ptr->msg_inj_tm /*- flit_ptr->pos*/);
              dmsg_total_network_latency   += /*(float)*/(sim_clock - flit_ptr->inj_tm + flit_ptr->e2e_penalty); 
              dmsg_total_latency           = cmsg_total_queueing_latency + cmsg_total_network_latency;
            }

#if TR_INTEG
            if((sim_clock - flit_ptr->inj_tm - flit_ptr->delay_cycle) < (HEAD_FLIT ? arch+1 : 3)*(flit_ptr->num_hop) - 1)
            {
              printf("ERR WEIRD ejecting at clock %lld flit_num:%d [%d][%d][%d] src %d, hops %d, time %d, delay %d enter time %lld\n", 
                  sim_clock,flit_ptr->flit_num, node, pc, i,flit_ptr->data.snode, flit_ptr->num_hop,
                  (int)(sim_clock - flit_ptr->inj_tm - flit_ptr->delay_cycle), (int)flit_ptr->delay_cycle, (long long)flit_ptr->msg_inj_tm);
              // exit(1);
              getchar();
            }
#endif
          }
          if(sql == YES)
            fprintf(fsql, "INSERT INTO FLOW VALUES(0, %d,%d,%d,%d,%d,%d);\n", 
                flit_ptr->flit_num, node, pc, vc_candidate, 5, sim_clock);

          if(verbose == YES) 
            //if(flit_ptr->priority_id == 0 && (HEAD_FLIT))
          { 
            printf("nic->PE [%d][%d][%d]-flit:%d(%s) source:%d ejt msgs:%d ejt flits:%d inj msgs:%d lat:%lld que:%d inj_tm:%d pri:%d pri id:%d\n", node, pc, vc_candidate, 
                flit_ptr->flit_num, (HEAD_FLIT)? "HEAD": (TAIL_FLIT)? "TAIL":"MIDDLE",
                flit_ptr->data.snode, num_ejt_msg,num_ejt_flit,num_inj_msg, sim_clock - flit_ptr->msg_inj_tm, flit_ptr->inj_tm - flit_ptr->msg_inj_tm, flit_ptr->msg_inj_tm,flit_ptr->priority,flit_ptr->priority_id);
            fflush(stdout);
          }

          //ejt_pc_utilization[node][pc]++;


          flit_free(flit_ptr);

#ifndef TR_INTEG
#ifndef TRACE_READ
          if(!source_queue_logic_enabled/* || active_source != -1*/)
            if(num_ejt_msg >= MSG_TO_EJECT + WARMUP_MSG)
            { 
              sim_end_flag = VALID; // End simulation. 
              return;	    
            }
#endif
#endif
        }// if (msg_cnt)
      }// for i
    }// for pc
  }// for node
} // nic_ejection_stage()
